{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f665dc49-6bc9-4d91-a32c-074252842501",
   "metadata": {},
   "outputs": [],
   "source": [
    "import openai\n",
    "import re\n",
    "import json\n",
    "import spacy\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "d316265a-bc4a-4d01-8ef4-abb0077ced87",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = json.load(open('MathQA/train.json'))\n",
    "dev_data = json.load(open('MathQA/dev.json'))\n",
    "dev_200_idx = np.load('MathQA/dev_200_idx.npy')\n",
    "dev_data_200 = []\n",
    "for i in dev_200_idx: dev_data_200.append(dev_data[i])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "7f7afddf-234a-4d5d-a842-9489f52c6b20",
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt_no_break = open(\"MathQA/prompt_no_break.txt\").read()\n",
    "prompt_explicit = open(\"MathQA/prompt_explicit.txt\").read()\n",
    "prompt_semicolon = open(\"MathQA/prompt_semicolon.txt\").read()\n",
    "prompt_exclamation = open(\"MathQA/prompt_exclamation.txt\").read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "056c3642-6a8c-4d5f-876d-36908461b172",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_ans_dict(c):\n",
    "    if(c.startswith(\"['\")):\n",
    "        c_ = eval(c)\n",
    "        ans_dict = {}\n",
    "        for ac in c_:\n",
    "            idx_, num_ = ac.split(' ) ')[-2:]\n",
    "            if('a' in idx_): ans_dict['a'] = num_\n",
    "            if('b' in idx_): ans_dict['b'] = num_\n",
    "            if('c' in idx_): ans_dict['c'] = num_\n",
    "            if('d' in idx_): ans_dict['d'] = num_\n",
    "            if('e' in idx_): ans_dict['e'] = num_\n",
    "    else:\n",
    "        ans_dict = {}\n",
    "        pattern = ' , [a-e] \\) '\n",
    "        c_ = re.split(pattern, c)\n",
    "        c_[0] = c_[0].split(' ) ')[-1]\n",
    "        for i, c in zip(['a', 'b', 'c', 'd', 'e'], c_):\n",
    "            ans_dict[i] = c\n",
    "    return ans_dict\n",
    "\n",
    "def test_answer(pred_str, ans_str):\n",
    "    \"\"\"Find the last number as the predicted answer\"\"\"\n",
    "    pattern = '\\d*\\.?\\d+'\n",
    "    pred = re.findall(pattern, pred_str)\n",
    "    if(len(pred) >= 1):\n",
    "        # print(pred_str)\n",
    "        pred = pred[-1]\n",
    "        gold = re.findall(pattern, ans_str)\n",
    "        # print(ans_str)\n",
    "        gold = gold[-1]\n",
    "        return pred == gold\n",
    "    else: return False\n",
    "\n",
    "def parse_pred_ans(filename):\n",
    "    with open(filename) as fd: lines = fd.readlines()\n",
    "    am, a = None, None\n",
    "    num_q, acc = 0, 0\n",
    "    current_mode = 'none'\n",
    "    questions = []\n",
    "    ans_pred = []\n",
    "    ans_gold = []\n",
    "    for l in lines:\n",
    "        if(l.startswith('Question: ')):\n",
    "            if(am is not None and a is not None):\n",
    "                questions.append(q)\n",
    "                ans_pred.append(am)\n",
    "                ans_gold.append(a)\n",
    "                if(test_answer(am, a)):\n",
    "                    acc += 1\n",
    "            current_mode = 'q'\n",
    "            q = l\n",
    "            num_q += 1\n",
    "        elif(l.startswith('Options: ')): pass\n",
    "        elif(l.startswith('A_model:')):\n",
    "            current_mode = 'am'\n",
    "            am = l\n",
    "        elif(l.startswith('A:')):\n",
    "            current_mode = 'a'\n",
    "            a = l\n",
    "        else:\n",
    "            if(current_mode == 'q'): q += l\n",
    "            elif(current_mode == 'am'): am += l\n",
    "            elif(current_mode == 'a'): a += l\n",
    "            elif(l == '\\n'): continue\n",
    "            else:\n",
    "                # print(q)\n",
    "                # print(am)\n",
    "                # print(a)\n",
    "                # print(l)\n",
    "                raise ValueError(current_mode)\n",
    "                \n",
    "    questions.append(q)\n",
    "    ans_pred.append(am)\n",
    "    ans_gold.append(a)\n",
    "    if(test_answer(am, a)):\n",
    "        acc += 1\n",
    "    print('num_q %d correct %d ratio %.4f' % (num_q, acc, float(acc / num_q)))\n",
    "    return questions, ans_pred, ans_gold"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "367409ba-74a1-4ec8-af67-14ed7f1dca1d",
   "metadata": {},
   "source": [
    "# Complex"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b0f553c8-142b-4519-b289-31fdf3b3fe8b",
   "metadata": {},
   "source": [
    "## No Break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "8b1f3b1a-425b-4f9d-ae89-43977b700a14",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 200/200 [13:38<00:00,  4.09s/it]\n"
     ]
    }
   ],
   "source": [
    "i = 0\n",
    "with open('outputs/dev_outputs_no_break.txt', 'w') as fd:\n",
    "    for case in tqdm(dev_data_200):\n",
    "        q = case['Problem']\n",
    "        c = case['options']\n",
    "        a = case['Rationale']\n",
    "        cr = case['correct']\n",
    "    \n",
    "        ans_dict = get_ans_dict(c)\n",
    "            \n",
    "        a += ' ' + ans_dict[cr]\n",
    "        \n",
    "        prompt_q = '' \n",
    "        prompt_q += '\\nQuestion: ' + q + '\\n'\n",
    "        prompt_q += 'Options: ' + c + '\\n'\n",
    "        prompt_q += \"Let's think step by step\\n\"\n",
    "        \n",
    "        prompt_full = ''.join(prompt_no_break) + prompt_q\n",
    "        response = openai.Completion.create(model=\"text-davinci-002\", \n",
    "                                            prompt=prompt_full, \n",
    "                                            temperature=0, \n",
    "                                            max_tokens=256)\n",
    "        \n",
    "        ans_model = response['choices'][0]['text']\n",
    "        fd.write('%sA_model:\\n%s\\nA:\\n%s\\n\\n' % (prompt_q, ans_model, a))\n",
    "        i += 1\n",
    "        # if(i == 2): break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "5e5b7cff-ce4f-42ed-b7dd-204c1d72de89",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Question: there were 35 students in a hostel . due to the admission of 7 new students the expenses of the mess were increased by rs . 84 per day while the average expenditure per head diminished by re 1 . what was the original expenditure of the mess ?\n",
      "Options: a ) rs 450 , b ) rs 920 , c ) rs 550 , d ) rs . 630 , e ) none of these\n",
      "Let's think step by step\n",
      "let the original average expenditure be rs . x. then , 42 ( x - 1 ) - 35 x = 84\\,7 x = 126. x = 18. therefore original expenditure = rs . ( 35 * 18 ) = rs . 630.\n",
      "The answer is d, rs . 630\n",
      "\n",
      "Question: a train 200 m long passes a man , running at 5 km / hr in the same direction in which the train is going , in 10 seconds . the speed of the train is ?\n",
      "Options: a ) 28 , b ) 50 , c ) 77 , d ) 22 , e ) 12\n",
      "Let's think step by step\n",
      "speed of the train relative to man = ( 200 / 10 ) m / sec = ( 20 ) m / sec. [ ( 20 ) * ( 18 / 5 ) ] km / hr = 72 km / hr. let the speed of the train be x km / hr. then , relative speed = ( x - 5 ) km / hr. x - 5 = 72, x = 77 km / hr .\n",
      "The answer is c, 77\n",
      "\n",
      "Question: solution x contains 20 % of material a and 80 % of material b . solution y contains 30 % of material a and 70 % of material b . a mixture of both these solutions contains 22 % of material a in the final product . how much solution x is present in the mixture ?\n",
      "Options: a ) 40 % , b ) 60 % , c ) 80 % , d ) 100 % , e ) 110 %\n",
      "Let's think step by step\n",
      "we can assume the total weight of the mixture = 100. conc of a in the final mixture = 22. let weight of a in the mixture be x. conc given = 20% = 0.2. therefore , weight of b = 100 - x. conc given = 30% = 0.3. now , accordding to the problem, 0.2 x + 0.3 ( 100 - x ) = 22. solving , we get x = 80. since we assumed the weight of the mixture = 100, therefore presence of a in the mixture = 80%.\n",
      "The answer is c, 80%\n",
      "\n",
      "Question: a trader sells 40 metres of cloth for rs . 8200 at a profit of rs . 35 per metre of cloth . how much profit will the trder earn on 40 metres of cloth ?\n",
      "Options: a ) rs . 950 , b ) rs . 1500 , c ) rs . 1000 , d ) rs . 1400 , e ) none of these\n",
      "Let's think step by step\n",
      "price of 1 metre cloth = 8200 / 40 = rs 205. cost of 1 metre cloth = rs 205 – 35 = rs 170. cost on 40 metres = 170 x 40 = rs . 6800. profit earned on 40 metres cloth = rs . 8200 – rs . 6800 = rs . 1400. \n",
      "The answer is d, rs . 1400\n",
      "\n",
      "Question: if x < y < z and y - x > 5 , where x is an even integer and y and z are odd integers , what is the least possible value s of z - x ?\n",
      "Options: a ) 6 , b ) 7 , c ) 8 , d ) 9 , e ) 10\n",
      "Let's think step by step\n",
      "We know x < y < z. to find the least possible value for z - x, we need to find the values for z and x that can be closest to each other. if x is some even number, then what could be minimum possible odd z. if x is some even number, y - x > 5 ; y > x + 5. minimum value for y = x + 5 + 2 = x + 7. (note : x + 5 is as even + odd = odd and nearest odd greater than x + 5 is x + 5 + 2). minimum value for z = y + 2 = x + 7 + 2 = x + 9. (note : z = y + 2 because both z and y are odd, difference between two odd numbers is 2). s = z - x = x + 9 - x = 9. \n",
      "The answer is d, 9\n",
      "\n",
      "Question: what is the difference between the c . i . on rs . 6000 for 1 1 / 2 years at 4 % per annum compounded yearly and half - yearly ?\n",
      "Options: a ) s . 2.04 , b ) s . 2.08 , c ) s . 2.02 , d ) s . 2.83 , e ) s . 2.45\n",
      "Let's think step by step\n",
      "c . i . when interest is compounded yearly = [ 6000 * ( 1 + 4 / 100 ) * ( 1 + ( 1 / 2 * 4 ) / 100 ] = 6000 * 26 / 25 * 51 / 50 = rs . 6364.8. c . i . when interest is compounded half - yearly = [ 6000 * ( 1 + 2 / 100 ) 2 ] = ( 6000 * 51 / 50 * 51 / 50 * 51 / 50 ) = rs . 6367.25. difference = ( 6367.25 - 6364.8 ) = rs . 2.45. \n",
      "The answer is e, s . 2.45\n",
      "\n",
      "Question: the average weight of a , b and c is 45 kg . if the average weight of a and b be 40 kg and that of b and c be 45 kg , then the weight of b is :\n",
      "Options: a ) 31 kg , b ) 32 kg , c ) 33 kg , d ) 35 kg , e ) none of these\n",
      "Let's think step by step\n",
      "let a , b , c represent their respective weights. then , we have : a + b + c = ( 45 x 3 ) = 135 … ( i ). a + b = ( 40 x 2 ) = 80 … ( ii ). b + c = ( 45 x 2 ) = 90 … ( iii ). adding ( ii ) and ( iii ), we get : a + 2 b + c = 170 … ( iv ). subtracting ( i ) from ( iv ), we get : b = 35. b’ s weight = 35 kg. \n",
      "The answer is d 35 kg\n",
      "\n",
      "Question: the compound and the simple interests on a certain sum at the same rate of interest for two years are rs . 11730 and rs . 10200 respectively . find the sum\n",
      "Options: a ) rs . 17037 , b ) rs . 17000 , c ) rs . 17276 , d ) rs . 170287 , e ) rs . 171881\n",
      "Let's think step by step\n",
      "the simple interest for the first year is 10200 / 2 is rs . 5100. and compound interest for first year also is rs . 5100. the compound interest for second year on rs . 5100 for one year. so rate of the interest = ( 100 * 1530 ) / ( 5100 * 1 ) = 30% p. so p = ( 100 * 10200 ) / ( 30 * 2 ) = rs . 17000. \n",
      "The answer is b, rs . 17000\n",
      "\n",
      "Question: amar takes as much time in running 22 meters as a car takes in covering 58 meters . what will be the distance covered by amar during the time the car covers 2.6 km ?\n",
      "Options: a ) 700 m , b ) 978 m , c ) 970 m , d ) 986 m , e ) 640 m\n",
      "Let's think step by step\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(prompt_full)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "cec60d13-d0b8-442f-b162-e9519685d88b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "num_q 200 correct 78 ratio 0.3900\n"
     ]
    }
   ],
   "source": [
    "_, _, _ = parse_pred_ans('outputs/dev_outputs_no_break.txt')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1628ddca-a640-4e13-b5f8-047890d3a5d3",
   "metadata": {},
   "source": [
    "## Explicit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "67dfd187-2200-44c0-b29b-0a8e128eafc2",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 200/200 [16:16<00:00,  4.88s/it]\n"
     ]
    }
   ],
   "source": [
    "i = 0\n",
    "with open('outputs/dev_outputs_explicit.txt', 'w') as fd:\n",
    "    for case in tqdm(dev_data_200):\n",
    "        q = case['Problem']\n",
    "        c = case['options']\n",
    "        a = case['Rationale']\n",
    "        cr = case['correct']\n",
    "    \n",
    "        ans_dict = get_ans_dict(c)\n",
    "            \n",
    "        a += ' ' + ans_dict[cr]\n",
    "        \n",
    "        prompt_q = '' \n",
    "        prompt_q += '\\nQuestion: ' + q + '\\n'\n",
    "        prompt_q += 'Options: ' + c + '\\n'\n",
    "        prompt_q += \"Let's think step by step\\n\"\n",
    "        \n",
    "        prompt_full = ''.join(prompt_explicit) + prompt_q\n",
    "        response = openai.Completion.create(model=\"text-davinci-002\", \n",
    "                                            prompt=prompt_full, \n",
    "                                            temperature=0, \n",
    "                                            max_tokens=256)\n",
    "        \n",
    "        ans_model = response['choices'][0]['text']\n",
    "        fd.write('%sA_model:\\n%s\\nA:\\n%s\\n\\n' % (prompt_q, ans_model, a))\n",
    "        i += 1\n",
    "        # if(i == 2): break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "281c888e-4f6d-4fd0-b5d9-8fe39f444f59",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Question: there were 35 students in a hostel . due to the admission of 7 new students the expenses of the mess were increased by rs . 84 per day while the average expenditure per head diminished by re 1 . what was the original expenditure of the mess ?\n",
      "Options: a ) rs 450 , b ) rs 920 , c ) rs 550 , d ) rs . 630 , e ) none of these\n",
      "Let's think step by step\n",
      "Step 1: let the original average expenditure be rs . x Step 2: then , 42 ( x - 1 ) - 35 x = 84\\,7 x = 126 Step 3: x = 18 Step 4: therefore original expenditure = rs . ( 35 * 18 ) = rs . 630.\n",
      "The answer is d, rs . 630\n",
      "\n",
      "Question: a train 200 m long passes a man , running at 5 km / hr in the same direction in which the train is going , in 10 seconds . the speed of the train is ?\n",
      "Options: a ) 28 , b ) 50 , c ) 77 , d ) 22 , e ) 12\n",
      "Let's think step by step\n",
      "Step 1: speed of the train relative to man = ( 200 / 10 ) m / sec = ( 20 ) m / sec. Step 2: [ ( 20 ) * ( 18 / 5 ) ] km / hr = 72 km / hr. Step 3: let the speed of the train be x km / hr. Step 4: then , relative speed = ( x - 5 ) km / hr. Step 5: x - 5 = 72, x = 77 km / hr .\n",
      "The answer is c, 77\n",
      "\n",
      "Question: solution x contains 20 % of material a and 80 % of material b . solution y contains 30 % of material a and 70 % of material b . a mixture of both these solutions contains 22 % of material a in the final product . how much solution x is present in the mixture ?\n",
      "Options: a ) 40 % , b ) 60 % , c ) 80 % , d ) 100 % , e ) 110 %\n",
      "Let's think step by step\n",
      "Step 1: we can assume the total weight of the mixture = 100. Step 2: conc of a in the final mixture = 22. Step 3: let weight of a in the mixture be x. Step 4: conc given = 20% = 0.2. Step 5: therefore , weight of b = 100 - x. Step 6: conc given = 30% = 0.3. Step 7: now , accordding to the problem, 0.2 x + 0.3 ( 100 - x ) = 22. Step 8: solving , we get x = 80. Step 9: since we assumed the weight of the mixture = 100, therefore presence of a in the mixture = 80%.\n",
      "The answer is c, 80%\n",
      "\n",
      "Question: a trader sells 40 metres of cloth for rs . 8200 at a profit of rs . 35 per metre of cloth . how much profit will the trder earn on 40 metres of cloth ?\n",
      "Options: a ) rs . 950 , b ) rs . 1500 , c ) rs . 1000 , d ) rs . 1400 , e ) none of these\n",
      "Let's think step by step\n",
      "Step 1: price of 1 metre cloth = 8200 / 40 = rs 205. Step 2: cost of 1 metre cloth = rs 205 – 35 = rs 170. Step 3: cost on 40 metres = 170 x 40 = rs . 6800. Step 4: profit earned on 40 metres cloth = rs . 8200 – rs . 6800 = rs . 1400\n",
      "The answer is d, rs . 1400\n",
      "\n",
      "Question: if x < y < z and y - x > 5 , where x is an even integer and y and z are odd integers , what is the least possible value s of z - x ?\n",
      "Options: a ) 6 , b ) 7 , c ) 8 , d ) 9 , e ) 10\n",
      "Let's think step by step\n",
      "Step 1: We know x < y < z. Step 2: to find the least possible value for z - x, we need to find the values for z and x that can be closest to each other. Step 3: if x is some even number, then what could be minimum possible odd z. Step 4: if x is some even number, y - x > 5 ; y > x + 5. Step 5: minimum value for y = x + 5 + 2 = x + 7. Step 6: (note : x + 5 is as even + odd = odd and nearest odd greater than x + 5 is x + 5 + 2). Step 7: minimum value for z = y + 2 = x + 7 + 2 = x + 9. Step 8: (note : z = y + 2 because both z and y are odd, difference between two odd numbers is 2). Step 9: s = z - x = x + 9 - x = 9\n",
      "The answer is d, 9\n",
      "\n",
      "Question: what is the difference between the c . i . on rs . 6000 for 1 1 / 2 years at 4 % per annum compounded yearly and half - yearly ?\n",
      "Options: a ) s . 2.04 , b ) s . 2.08 , c ) s . 2.02 , d ) s . 2.83 , e ) s . 2.45\n",
      "Let's think step by step\n",
      "Step 1: c . i . when interest is compounded yearly = [ 6000 * ( 1 + 4 / 100 ) * ( 1 + ( 1 / 2 * 4 ) / 100 ] = 6000 * 26 / 25 * 51 / 50 = rs . 6364.8. Step 2: c . i . when interest is compounded half - yearly = [ 6000 * ( 1 + 2 / 100 ) 2 ] = ( 6000 * 51 / 50 * 51 / 50 * 51 / 50 ) = rs . 6367.25. Step 3: difference = ( 6367.25 - 6364.8 ) = rs . 2.45.\n",
      "The answer is e, s . 2.45\n",
      "\n",
      "Question: the average weight of a , b and c is 45 kg . if the average weight of a and b be 40 kg and that of b and c be 45 kg , then the weight of b is :\n",
      "Options: a ) 31 kg , b ) 32 kg , c ) 33 kg , d ) 35 kg , e ) none of these\n",
      "Let's think step by step\n",
      "Step 1: let a , b , c represent their respective weights. Step 2: then , we have : a + b + c = ( 45 x 3 ) = 135 … ( i ) Step 3: a + b = ( 40 x 2 ) = 80 … ( ii ) Step 4: b + c = ( 45 x 2 ) = 90 … ( iii ) Step 5: adding ( ii ) and ( iii ), we get : a + 2 b + c = 170 … ( iv ) Step 6: subtracting ( i ) from ( iv ), we get : b = 35. Step 7: b’ s weight = 35 kg .\n",
      "The answer is d 35 kg\n",
      "\n",
      "Question: the compound and the simple interests on a certain sum at the same rate of interest for two years are rs . 11730 and rs . 10200 respectively . find the sum\n",
      "Options: a ) rs . 17037 , b ) rs . 17000 , c ) rs . 17276 , d ) rs . 170287 , e ) rs . 171881\n",
      "Let's think step by step\n",
      "Step 1: the simple interest for the first year is 10200 / 2 is rs . 5100. Step 2: and compound interest for first year also is rs . 5100. Step 3: the compound interest for second year on rs . 5100 for one year. Step 4: so rate of the interest = ( 100 * 1530 ) / ( 5100 * 1 ) = 30% p. Step 5: so p = ( 100 * 10200 ) / ( 30 * 2 ) = rs . 17000\n",
      "The answer is b, rs . 17000\n",
      "\n",
      "Question: average of 5 numbers is 20 , and the sum of 3 of the numbers is 48 , what is the average of the other 2 numbers ?\n",
      "Options: a ) 24 , b ) 34 , c ) 29 , d ) 35 , e ) 26\n",
      "Let's think step by step\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(prompt_full)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "143bafc0-bdac-4984-a295-ebfc74033e7e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "num_q 200 correct 72 ratio 0.3600\n"
     ]
    }
   ],
   "source": [
    "_, _, _ = parse_pred_ans('outputs/dev_outputs_explicit.txt')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d5b1d2ca-4da5-43b0-9772-b389e83fd152",
   "metadata": {},
   "source": [
    "## Semicolon"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "5605fb56-71d0-4ff3-a57a-17453b8bea3a",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 200/200 [14:00<00:00,  4.20s/it]\n"
     ]
    }
   ],
   "source": [
    "i = 0\n",
    "with open('outputs/dev_outputs_semicolon.txt', 'w') as fd:\n",
    "    for case in tqdm(dev_data_200):\n",
    "        q = case['Problem']\n",
    "        c = case['options']\n",
    "        a = case['Rationale']\n",
    "        cr = case['correct']\n",
    "    \n",
    "        ans_dict = get_ans_dict(c)\n",
    "            \n",
    "        a += ' ' + ans_dict[cr]\n",
    "        \n",
    "        prompt_q = '' \n",
    "        prompt_q += '\\nQuestion: ' + q + '\\n'\n",
    "        prompt_q += 'Options: ' + c + '\\n'\n",
    "        prompt_q += \"Let's think step by step\\n\"\n",
    "        \n",
    "        prompt_full = ''.join(prompt_semicolon) + prompt_q\n",
    "        response = openai.Completion.create(model=\"text-davinci-002\", \n",
    "                                            prompt=prompt_full, \n",
    "                                            temperature=0, \n",
    "                                            max_tokens=256)\n",
    "        \n",
    "        ans_model = response['choices'][0]['text']\n",
    "        fd.write('%sA_model:\\n%s\\nA:\\n%s\\n\\n' % (prompt_q, ans_model, a))\n",
    "        i += 1\n",
    "        # if(i == 2): break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "7f2decfd-a7e5-4c8b-aa8d-46579f51b373",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Question: there were 35 students in a hostel . due to the admission of 7 new students the expenses of the mess were increased by rs . 84 per day while the average expenditure per head diminished by re 1 . what was the original expenditure of the mess ?\n",
      "Options: a ) rs 450 , b ) rs 920 , c ) rs 550 , d ) rs . 630 , e ) none of these\n",
      "Let's think step by step\n",
      "let the original average expenditure be rs . x; then , 42 ( x - 1 ) - 35 x = 84\\,7 x = 126; x = 18; therefore original expenditure = rs . ( 35 * 18 ) = rs . 630; \n",
      "The answer is d, rs . 630\n",
      "\n",
      "Question: a train 200 m long passes a man , running at 5 km / hr in the same direction in which the train is going , in 10 seconds . the speed of the train is ?\n",
      "Options: a ) 28 , b ) 50 , c ) 77 , d ) 22 , e ) 12\n",
      "Let's think step by step\n",
      "speed of the train relative to man = ( 200 / 10 ) m / sec = ( 20 ) m / sec; [ ( 20 ) * ( 18 / 5 ) ] km / hr = 72 km / hr; let the speed of the train be x km / hr; then , relative speed = ( x - 5 ) km / hr; x - 5 = 72, x = 77 km / hr; \n",
      "The answer is c, 77\n",
      "\n",
      "Question: solution x contains 20 % of material a and 80 % of material b . solution y contains 30 % of material a and 70 % of material b . a mixture of both these solutions contains 22 % of material a in the final product . how much solution x is present in the mixture ?\n",
      "Options: a ) 40 % , b ) 60 % , c ) 80 % , d ) 100 % , e ) 110 %\n",
      "Let's think step by step\n",
      "we can assume the total weight of the mixture = 100; conc of a in the final mixture = 22; let weight of a in the mixture be x; conc given = 20% = 0.2; therefore , weight of b = 100 - x; conc given = 30% = 0.3; now , accordding to the problem, 0.2 x + 0.3 ( 100 - x ) = 22; solving , we get x = 80; since we assumed the weight of the mixture = 100, therefore presence of a in the mixture = 80%; \n",
      "The answer is c, 80%\n",
      "\n",
      "Question: a trader sells 40 metres of cloth for rs . 8200 at a profit of rs . 35 per metre of cloth . how much profit will the trder earn on 40 metres of cloth ?\n",
      "Options: a ) rs . 950 , b ) rs . 1500 , c ) rs . 1000 , d ) rs . 1400 , e ) none of these\n",
      "Let's think step by step\n",
      "price of 1 metre cloth = 8200 / 40 = rs 205; cost of 1 metre cloth = rs 205 – 35 = rs 170; cost on 40 metres = 170 x 40 = rs . 6800; profit earned on 40 metres cloth = rs . 8200 – rs . 6800 = rs . 1400; \n",
      "The answer is d, rs . 1400\n",
      "\n",
      "Question: if x < y < z and y - x > 5 , where x is an even integer and y and z are odd integers , what is the least possible value s of z - x ?\n",
      "Options: a ) 6 , b ) 7 , c ) 8 , d ) 9 , e ) 10\n",
      "Let's think step by step\n",
      "We know x < y < z; to find the least possible value for z - x, we need to find the values for z and x that can be closest to each other; if x is some even number, then what could be minimum possible odd z; if x is some even number, y - x > 5 ; y > x + 5; minimum value for y = x + 5 + 2 = x + 7; (note : x + 5 is as even + odd = odd and nearest odd greater than x + 5 is x + 5 + 2); minimum value for z = y + 2 = x + 7 + 2 = x + 9; (note : z = y + 2 because both z and y are odd, difference between two odd numbers is 2); s = z - x = x + 9 - x = 9; \n",
      "The answer is d, 9\n",
      "\n",
      "Question: what is the difference between the c . i . on rs . 6000 for 1 1 / 2 years at 4 % per annum compounded yearly and half - yearly ?\n",
      "Options: a ) s . 2.04 , b ) s . 2.08 , c ) s . 2.02 , d ) s . 2.83 , e ) s . 2.45\n",
      "Let's think step by step\n",
      "c . i . when interest is compounded yearly = [ 6000 * ( 1 + 4 / 100 ) * ( 1 + ( 1 / 2 * 4 ) / 100 ] = 6000 * 26 / 25 * 51 / 50 = rs . 6364.8; c . i . when interest is compounded half - yearly = [ 6000 * ( 1 + 2 / 100 ) 2 ] = ( 6000 * 51 / 50 * 51 / 50 * 51 / 50 ) = rs . 6367.25; difference = ( 6367.25 - 6364.8 ) = rs . 2.45; \n",
      "The answer is e, s . 2.45\n",
      "\n",
      "Question: the average weight of a , b and c is 45 kg . if the average weight of a and b be 40 kg and that of b and c be 45 kg , then the weight of b is :\n",
      "Options: a ) 31 kg , b ) 32 kg , c ) 33 kg , d ) 35 kg , e ) none of these\n",
      "Let's think step by step\n",
      "let a , b , c represent their respective weights; then , we have : a + b + c = ( 45 x 3 ) = 135 … ( i ); a + b = ( 40 x 2 ) = 80 … ( ii ); b + c = ( 45 x 2 ) = 90 … ( iii ); adding ( ii ) and ( iii ), we get : a + 2 b + c = 170 … ( iv ); subtracting ( i ) from ( iv ), we get : b = 35; b’ s weight = 35 kg; \n",
      "The answer is d 35 kg\n",
      "\n",
      "Question: the compound and the simple interests on a certain sum at the same rate of interest for two years are rs . 11730 and rs . 10200 respectively . find the sum\n",
      "Options: a ) rs . 17037 , b ) rs . 17000 , c ) rs . 17276 , d ) rs . 170287 , e ) rs . 171881\n",
      "Let's think step by step\n",
      "the simple interest for the first year is 10200 / 2 is rs . 5100; and compound interest for first year also is rs . 5100; the compound interest for second year on rs . 5100 for one year; so rate of the interest = ( 100 * 1530 ) / ( 5100 * 1 ) = 30% p; so p = ( 100 * 10200 ) / ( 30 * 2 ) = rs . 17000; \n",
      "The answer is b, rs . 17000\n",
      "\n",
      "Question: average of 5 numbers is 20 , and the sum of 3 of the numbers is 48 , what is the average of the other 2 numbers ?\n",
      "Options: a ) 24 , b ) 34 , c ) 29 , d ) 35 , e ) 26\n",
      "Let's think step by step\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(prompt_full)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "4e570b27-4aa3-4017-b65e-21fbb8358a92",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "num_q 200 correct 79 ratio 0.3950\n"
     ]
    }
   ],
   "source": [
    "_, _, _ = parse_pred_ans('outputs/dev_outputs_semicolon.txt')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "37a941c9-8d7d-4b2b-bd98-1dd2fd97bee4",
   "metadata": {},
   "source": [
    "## Exclamation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "760d3151-2cc2-4a31-83ed-35febe8ea900",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 200/200 [14:27<00:00,  4.34s/it]\n"
     ]
    }
   ],
   "source": [
    "i = 0\n",
    "with open('outputs/dev_outputs_exclamation.txt', 'w') as fd:\n",
    "    for case in tqdm(dev_data_200):\n",
    "        q = case['Problem']\n",
    "        c = case['options']\n",
    "        a = case['Rationale']\n",
    "        cr = case['correct']\n",
    "    \n",
    "        ans_dict = get_ans_dict(c)\n",
    "            \n",
    "        a += ' ' + ans_dict[cr]\n",
    "        \n",
    "        prompt_q = '' \n",
    "        prompt_q += '\\nQuestion: ' + q + '\\n'\n",
    "        prompt_q += 'Options: ' + c + '\\n'\n",
    "        prompt_q += \"Let's think step by step\\n\"\n",
    "        \n",
    "        prompt_full = ''.join(prompt_exclamation) + prompt_q\n",
    "        response = openai.Completion.create(model=\"text-davinci-002\", \n",
    "                                            prompt=prompt_full, \n",
    "                                            temperature=0, \n",
    "                                            max_tokens=256)\n",
    "        \n",
    "        ans_model = response['choices'][0]['text']\n",
    "        fd.write('%sA_model:\\n%s\\nA:\\n%s\\n\\n' % (prompt_q, ans_model, a))\n",
    "        i += 1\n",
    "        # if(i == 2): break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "00a5fe85-35a1-44dc-ab36-a69de55f9386",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Question: there were 35 students in a hostel . due to the admission of 7 new students the expenses of the mess were increased by rs . 84 per day while the average expenditure per head diminished by re 1 . what was the original expenditure of the mess ?\n",
      "Options: a ) rs 450 , b ) rs 920 , c ) rs 550 , d ) rs . 630 , e ) none of these\n",
      "Let's think step by step\n",
      "let the original average expenditure be rs . x!!! then , 42 ( x - 1 ) - 35 x = 84\\,7 x = 126!!! x = 18!!! therefore original expenditure = rs . ( 35 * 18 ) = rs . 630!!! \n",
      "The answer is d, rs . 630\n",
      "\n",
      "Question: a train 200 m long passes a man , running at 5 km / hr in the same direction in which the train is going , in 10 seconds . the speed of the train is ?\n",
      "Options: a ) 28 , b ) 50 , c ) 77 , d ) 22 , e ) 12\n",
      "Let's think step by step\n",
      "speed of the train relative to man = ( 200 / 10 ) m / sec = ( 20 ) m / sec!!! [ ( 20 ) * ( 18 / 5 ) ] km / hr = 72 km / hr!!! let the speed of the train be x km / hr!!! then , relative speed = ( x - 5 ) km / hr!!! x - 5 = 72, x = 77 km / hr!!! \n",
      "The answer is c, 77\n",
      "\n",
      "Question: solution x contains 20 % of material a and 80 % of material b . solution y contains 30 % of material a and 70 % of material b . a mixture of both these solutions contains 22 % of material a in the final product . how much solution x is present in the mixture ?\n",
      "Options: a ) 40 % , b ) 60 % , c ) 80 % , d ) 100 % , e ) 110 %\n",
      "Let's think step by step\n",
      "we can assume the total weight of the mixture = 100!!! conc of a in the final mixture = 22!!! let weight of a in the mixture be x!!! conc given = 20% = 0.2!!! therefore , weight of b = 100 - x!!! conc given = 30% = 0.3!!! now , accordding to the problem, 0.2 x + 0.3 ( 100 - x ) = 22!!! solving , we get x = 80!!! since we assumed the weight of the mixture = 100, therefore presence of a in the mixture = 80%!!! \n",
      "The answer is c, 80%\n",
      "\n",
      "Question: a trader sells 40 metres of cloth for rs . 8200 at a profit of rs . 35 per metre of cloth . how much profit will the trder earn on 40 metres of cloth ?\n",
      "Options: a ) rs . 950 , b ) rs . 1500 , c ) rs . 1000 , d ) rs . 1400 , e ) none of these\n",
      "Let's think step by step\n",
      "price of 1 metre cloth = 8200 / 40 = rs 205!!! cost of 1 metre cloth = rs 205 – 35 = rs 170!!! cost on 40 metres = 170 x 40 = rs . 6800!!! profit earned on 40 metres cloth = rs . 8200 – rs . 6800 = rs . 1400!!! \n",
      "The answer is d, rs . 1400\n",
      "\n",
      "Question: if x < y < z and y - x > 5 , where x is an even integer and y and z are odd integers , what is the least possible value s of z - x ?\n",
      "Options: a ) 6 , b ) 7 , c ) 8 , d ) 9 , e ) 10\n",
      "Let's think step by step\n",
      "We know x < y < z!!! to find the least possible value for z - x, we need to find the values for z and x that can be closest to each other!!! if x is some even number, then what could be minimum possible odd z!!! if x is some even number, y - x > 5 !!! y > x + 5!!! minimum value for y = x + 5 + 2 = x + 7!!! (note : x + 5 is as even + odd = odd and nearest odd greater than x + 5 is x + 5 + 2)!!! minimum value for z = y + 2 = x + 7 + 2 = x + 9!!! (note : z = y + 2 because both z and y are odd, difference between two odd numbers is 2)!!! s = z - x = x + 9 - x = 9!!! \n",
      "The answer is d, 9\n",
      "\n",
      "Question: what is the difference between the c . i . on rs . 6000 for 1 1 / 2 years at 4 % per annum compounded yearly and half - yearly ?\n",
      "Options: a ) s . 2.04 , b ) s . 2.08 , c ) s . 2.02 , d ) s . 2.83 , e ) s . 2.45\n",
      "Let's think step by step\n",
      "c . i . when interest is compounded yearly = [ 6000 * ( 1 + 4 / 100 ) * ( 1 + ( 1 / 2 * 4 ) / 100 ] = 6000 * 26 / 25 * 51 / 50 = rs . 6364.8!!! c . i . when interest is compounded half - yearly = [ 6000 * ( 1 + 2 / 100 ) 2 ] = ( 6000 * 51 / 50 * 51 / 50 * 51 / 50 ) = rs . 6367.25!!! difference = ( 6367.25 - 6364.8 ) = rs . 2.45!!! \n",
      "The answer is e, s . 2.45\n",
      "\n",
      "Question: the average weight of a , b and c is 45 kg . if the average weight of a and b be 40 kg and that of b and c be 45 kg , then the weight of b is :\n",
      "Options: a ) 31 kg , b ) 32 kg , c ) 33 kg , d ) 35 kg , e ) none of these\n",
      "Let's think step by step\n",
      "let a , b , c represent their respective weights!!! then , we have : a + b + c = ( 45 x 3 ) = 135 … ( i )!!! a + b = ( 40 x 2 ) = 80 … ( ii )!!! b + c = ( 45 x 2 ) = 90 … ( iii )!!! adding ( ii ) and ( iii ), we get : a + 2 b + c = 170 … ( iv )!!! subtracting ( i ) from ( iv ), we get : b = 35!!! b’ s weight = 35 kg!!! \n",
      "The answer is d 35 kg\n",
      "\n",
      "Question: the compound and the simple interests on a certain sum at the same rate of interest for two years are rs . 11730 and rs . 10200 respectively . find the sum\n",
      "Options: a ) rs . 17037 , b ) rs . 17000 , c ) rs . 17276 , d ) rs . 170287 , e ) rs . 171881\n",
      "Let's think step by step\n",
      "the simple interest for the first year is 10200 / 2 is rs . 5100!!! and compound interest for first year also is rs . 5100!!! the compound interest for second year on rs . 5100 for one year!!! so rate of the interest = ( 100 * 1530 ) / ( 5100 * 1 ) = 30% p!!! so p = ( 100 * 10200 ) / ( 30 * 2 ) = rs . 17000!!! \n",
      "The answer is b, rs . 17000\n",
      "\n",
      "Question: average of 5 numbers is 20 , and the sum of 3 of the numbers is 48 , what is the average of the other 2 numbers ?\n",
      "Options: a ) 24 , b ) 34 , c ) 29 , d ) 35 , e ) 26\n",
      "Let's think step by step\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(prompt_full)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "5e2419d4-bd35-445b-bae6-7d5b25df22e0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "num_q 200 correct 79 ratio 0.3950\n"
     ]
    }
   ],
   "source": [
    "_, _, _ = parse_pred_ans('outputs/dev_outputs_exclamation.txt')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5d32aa47-382d-443d-ac2e-47b51ce6163d",
   "metadata": {},
   "source": [
    "# Simple"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dd499ddf-cabc-4d5a-846a-d33fa1753510",
   "metadata": {},
   "source": [
    "## No Break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "0f6e9d9f-aa6b-42d9-9cb9-b8c9c0ef75cc",
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt_simple_no_break = open('MathQA/prompt_simple_no_break.txt').read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "e1a47e7d-6953-4630-9a01-e25146fefe7e",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Question: on a purchase of $ 130 , a store offered a payment plan consisting of a $ 30 down payment and 12 monthly payments of $ 10 each . what percent of the purchase price , to the nearest tenth of a percent , did the customer pay in interest by using this plan ?\n",
      "Options: a ) 23.5 % , b ) 24.5 % , c ) 23.1 % , d ) 25.5 % , e ) 26.5 %\n",
      "Let's think step by step\n",
      "12 * 10 + 30 = 150. ( 30 / 130 ) * 100 = 23.1\n",
      "The answer is c, 23.1 %\n",
      "\n",
      "Question: the ratio of three numbers is 1 : 3 : 4 and the sum of their squares is 2600 . the sum of the numbers is ?\n",
      "Options: a ) 10 , b ) 12 , c ) 15 , d ) 14 , e ) 9\n",
      "Let's think step by step\n",
      "let the numbers be x , 3 x , 4 x then , x^2 + 9x^2 + 16x^2 = 2600. 26 x^2 = 2600, x^2 = 100, x = 10\n",
      "The answer is a, 10\n",
      "\n",
      "Question: the average weight of 8 person ' s increases by 1.5 kg when a new person comes in place of one of them weighing 75 kg . what might be the weight of the new person ?\n",
      "Options: a ) 81 kg , b ) 85 kg , c ) 90 kg , d ) 100 kg , e ) 110 kg\n",
      "Let's think step by step\n",
      "total weight increased = ( 8 x 1.5 ) kg = 6 kg. weight of new person = ( 75 + 6 ) kg = 81 kg.\n",
      "The answer is a, 81 kg\n",
      "\n",
      "Question: how many positive integers less than 60 have a reminder 01 when divided by 3 ?\n",
      "Options: a ) 13 , b ) 14 , c ) 15 , d ) 16 , e ) 20\n",
      "Let's think step by step\n",
      "1 also gives the remainder of 1 when divided by 3. so , there are total of 20 numbers .\n",
      "The answer is e, 20\n",
      "\n",
      "Question: 8 is 2 % of a , and 2 is 8 % of b . c equals b / a . what is the value of c ?\n",
      "Options: a ) 1 / 16 , b ) 1 / 4 , c ) 1 , d ) 4 , e ) 8\n",
      "Let's think step by step\n",
      "2a / 100 = 8, a = 400. 8b / 100 = 2, b = 25. c = b / a = 25 / 400 = 1 / 16\n",
      "The answer is a, 1 / 16\n",
      "\n",
      "Question: the membership of a committee consists of 3 english teachers , 4 mathematics teachers , and 2 social studies teachers . if 2 committee members are to be selected at random to write the committee ’ s report , what is the probability that the two members selected will both be maths teachers ?\n",
      "Options: a ) 2 / 3 , b ) 1 / 3 , c ) 2 / 9 , d ) 1 / 24 , e ) 1 / 12\n",
      "Let's think step by step\n",
      "probability of first member an english teacher = 3 / 9. probability of second member an english teacher = 2 / 8. probability of both being english teacher = 3 / 9 x 2 / 8 = 1 / 12\n",
      "The answer is e, 1 / 12\n",
      "\n",
      "Question: the radius of a cylindrical vessel is 7 cm and height is 5 cm . find the whole surface of the cylinder ?\n",
      "Options: a ) 281 , b ) 528 , c ) 440 , d ) 767 , e ) 1981\n",
      "Let's think step by step\n",
      "r = 7, h = 5. 2 π r ( h + r ) = 2 * 22 / 7 * 7 ( 12 ) = 528\n",
      "The answer is b, 528\n",
      "\n",
      "Question: dan ’ s car gets 32 miles per gallon . if gas costs $ 4 / gallon , then how many miles can dan ’ s car go on $ 58 of gas ?\n",
      "Options: a ) 242 , b ) 353 , c ) 464 , d ) 575 , e ) 686\n",
      "Let's think step by step\n",
      "58 / 4 = 14.5 gallons. 14.5 * 32 = 464 miles\n",
      "the answer is c, 464\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(prompt_simple_no_break)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "263c8153-e37e-45eb-b0aa-5eb76f98b5fe",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 200/200 [09:42<00:00,  2.91s/it]\n"
     ]
    }
   ],
   "source": [
    "i = 0\n",
    "with open('outputs/dev_outputs_simple_no_break.txt', 'w') as fd:\n",
    "    for case in tqdm(dev_data_200):\n",
    "        q = case['Problem']\n",
    "        c = case['options']\n",
    "        a = case['Rationale']\n",
    "        cr = case['correct']\n",
    "    \n",
    "        ans_dict = get_ans_dict(c)\n",
    "            \n",
    "        a += ' ' + ans_dict[cr]\n",
    "        \n",
    "        prompt_q = '' \n",
    "        prompt_q += '\\nQuestion: ' + q + '\\n'\n",
    "        prompt_q += 'Options: ' + c + '\\n'\n",
    "        prompt_q += \"Let's think step by step\\n\"\n",
    "        \n",
    "        prompt_full = ''.join(prompt_simple_no_break) + prompt_q\n",
    "        response = openai.Completion.create(model=\"text-davinci-002\", \n",
    "                                            prompt=prompt_full, \n",
    "                                            temperature=0, \n",
    "                                            max_tokens=256)\n",
    "        \n",
    "        ans_model = response['choices'][0]['text']\n",
    "        fd.write('%sA_model:\\n%s\\nA:\\n%s\\n\\n' % (prompt_q, ans_model, a))\n",
    "        i += 1\n",
    "        # if(i == 2): break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "70358df3-d5a0-4f27-8303-1bdb1f2462c3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "num_q 200 correct 69 ratio 0.3450\n"
     ]
    }
   ],
   "source": [
    "_, _, _ = parse_pred_ans('outputs/dev_outputs_simple_no_break.txt')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c2a52aa0-3bfe-4f0a-8925-3da87f8234dc",
   "metadata": {},
   "source": [
    "## Explicit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "6d897c22-f019-4808-a57d-97fb0dc21d6a",
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt_simple_explicit = open('MathQA/prompt_simple_explicit.txt').read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "82e73681-484f-4ba8-a2e7-ff31d303e81a",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Question: on a purchase of $ 130 , a store offered a payment plan consisting of a $ 30 down payment and 12 monthly payments of $ 10 each . what percent of the purchase price , to the nearest tenth of a percent , did the customer pay in interest by using this plan ?\n",
      "Options: a ) 23.5 % , b ) 24.5 % , c ) 23.1 % , d ) 25.5 % , e ) 26.5 %\n",
      "Let's think step by step\n",
      "Step 1: 12 * 10 + 30 = 150. Step 2: ( 30 / 130 ) * 100 = 23.1\n",
      "The answer is c, 23.1 %\n",
      "\n",
      "Question: the ratio of three numbers is 1 : 3 : 4 and the sum of their squares is 2600 . the sum of the numbers is ?\n",
      "Options: a ) 10 , b ) 12 , c ) 15 , d ) 14 , e ) 9\n",
      "Let's think step by step\n",
      "Step 1: let the numbers be x , 3 x , 4 x then , x^2 + 9x^2 + 16x^2 = 2600. Step 2: 26 x^2 = 2600, x^2 = 100, x = 10\n",
      "The answer is a, 10\n",
      "\n",
      "Question: the average weight of 8 person ' s increases by 1.5 kg when a new person comes in place of one of them weighing 75 kg . what might be the weight of the new person ?\n",
      "Options: a ) 81 kg , b ) 85 kg , c ) 90 kg , d ) 100 kg , e ) 110 kg\n",
      "Let's think step by step\n",
      "Step 1: total weight increased = ( 8 x 1.5 ) kg = 6 kg. Step 2: weight of new person = ( 75 + 6 ) kg = 81 kg.\n",
      "The answer is a, 81 kg\n",
      "\n",
      "Question: how many positive integers less than 60 have a reminder 01 when divided by 3 ?\n",
      "Options: a ) 13 , b ) 14 , c ) 15 , d ) 16 , e ) 20\n",
      "Let's think step by step\n",
      "Step 1: 1 also gives the remainder of 1 when divided by 3. Step 2: so , there are total of 20 numbers .\n",
      "The answer is e, 20\n",
      "\n",
      "Question: 8 is 2 % of a , and 2 is 8 % of b . c equals b / a . what is the value of c ?\n",
      "Options: a ) 1 / 16 , b ) 1 / 4 , c ) 1 , d ) 4 , e ) 8\n",
      "Let's think step by step\n",
      "Step 1: 2a / 100 = 8, a = 400. Step 2: 8b / 100 = 2, b = 25. Step 3: c = b / a = 25 / 400 = 1 / 16\n",
      "The answer is a, 1 / 16\n",
      "\n",
      "Question: the membership of a committee consists of 3 english teachers , 4 mathematics teachers , and 2 social studies teachers . if 2 committee members are to be selected at random to write the committee ’ s report , what is the probability that the two members selected will both be maths teachers ?\n",
      "Options: a ) 2 / 3 , b ) 1 / 3 , c ) 2 / 9 , d ) 1 / 24 , e ) 1 / 12\n",
      "Let's think step by step\n",
      "Step 1: probability of first member an english teacher = 3 / 9. Step 2: probability of second member an english teacher = 2 / 8.. Step 3: probability of both being english teacher = 3 / 9 x 2 / 8 = 1 / 12\n",
      "The answer is e, 1 / 12\n",
      "\n",
      "Question: the radius of a cylindrical vessel is 7 cm and height is 5 cm . find the whole surface of the cylinder ?\n",
      "Options: a ) 281 , b ) 528 , c ) 440 , d ) 767 , e ) 1981\n",
      "Let's think step by step\n",
      "Step 1: r = 7, h = 5; Step 2: 2 π r ( h + r ) = 2 * 22 / 7 * 7 ( 12 ) = 528\n",
      "The answer is b, 528\n",
      "\n",
      "Question: dan ’ s car gets 32 miles per gallon . if gas costs $ 4 / gallon , then how many miles can dan ’ s car go on $ 58 of gas ?\n",
      "Options: a ) 242 , b ) 353 , c ) 464 , d ) 575 , e ) 686\n",
      "Let's think step by step\n",
      "Step 1: 58 / 4 = 14.5 gallons. Step 2: 14.5 * 32 = 464 miles\n",
      "the answer is c, 464\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(prompt_simple_explicit)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "5d2949b8-aaae-4930-8581-6e33db989312",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 200/200 [10:45<00:00,  3.23s/it]\n"
     ]
    }
   ],
   "source": [
    "i = 0\n",
    "with open('outputs/dev_outputs_simple_explicit.txt', 'w') as fd:\n",
    "    for case in tqdm(dev_data_200):\n",
    "        q = case['Problem']\n",
    "        c = case['options']\n",
    "        a = case['Rationale']\n",
    "        cr = case['correct']\n",
    "    \n",
    "        ans_dict = get_ans_dict(c)\n",
    "            \n",
    "        a += ' ' + ans_dict[cr]\n",
    "        \n",
    "        prompt_q = '' \n",
    "        prompt_q += '\\nQuestion: ' + q + '\\n'\n",
    "        prompt_q += 'Options: ' + c + '\\n'\n",
    "        prompt_q += \"Let's think step by step\\n\"\n",
    "        \n",
    "        prompt_full = ''.join(prompt_simple_explicit) + prompt_q\n",
    "        response = openai.Completion.create(model=\"text-davinci-002\", \n",
    "                                            prompt=prompt_full, \n",
    "                                            temperature=0, \n",
    "                                            max_tokens=256)\n",
    "        \n",
    "        ans_model = response['choices'][0]['text']\n",
    "        fd.write('%sA_model:\\n%s\\nA:\\n%s\\n\\n' % (prompt_q, ans_model, a))\n",
    "        i += 1\n",
    "        # if(i == 2): break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "c4cf7c7f-64f9-48fa-8895-0240616d223d",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Question: on a purchase of $ 130 , a store offered a payment plan consisting of a $ 30 down payment and 12 monthly payments of $ 10 each . what percent of the purchase price , to the nearest tenth of a percent , did the customer pay in interest by using this plan ?\n",
      "Options: a ) 23.5 % , b ) 24.5 % , c ) 23.1 % , d ) 25.5 % , e ) 26.5 %\n",
      "Let's think step by step\n",
      "Step 1: 12 * 10 + 30 = 150. Step 2: ( 30 / 130 ) * 100 = 23.1\n",
      "The answer is c, 23.1 %\n",
      "\n",
      "Question: the ratio of three numbers is 1 : 3 : 4 and the sum of their squares is 2600 . the sum of the numbers is ?\n",
      "Options: a ) 10 , b ) 12 , c ) 15 , d ) 14 , e ) 9\n",
      "Let's think step by step\n",
      "Step 1: let the numbers be x , 3 x , 4 x then , x^2 + 9x^2 + 16x^2 = 2600. Step 2: 26 x^2 = 2600, x^2 = 100, x = 10\n",
      "The answer is a, 10\n",
      "\n",
      "Question: the average weight of 8 person ' s increases by 1.5 kg when a new person comes in place of one of them weighing 75 kg . what might be the weight of the new person ?\n",
      "Options: a ) 81 kg , b ) 85 kg , c ) 90 kg , d ) 100 kg , e ) 110 kg\n",
      "Let's think step by step\n",
      "Step 1: total weight increased = ( 8 x 1.5 ) kg = 6 kg. Step 2: weight of new person = ( 75 + 6 ) kg = 81 kg.\n",
      "The answer is a, 81 kg\n",
      "\n",
      "Question: how many positive integers less than 60 have a reminder 01 when divided by 3 ?\n",
      "Options: a ) 13 , b ) 14 , c ) 15 , d ) 16 , e ) 20\n",
      "Let's think step by step\n",
      "Step 1: 1 also gives the remainder of 1 when divided by 3. Step 2: so , there are total of 20 numbers .\n",
      "The answer is e, 20\n",
      "\n",
      "Question: 8 is 2 % of a , and 2 is 8 % of b . c equals b / a . what is the value of c ?\n",
      "Options: a ) 1 / 16 , b ) 1 / 4 , c ) 1 , d ) 4 , e ) 8\n",
      "Let's think step by step\n",
      "Step 1: 2a / 100 = 8, a = 400. Step 2: 8b / 100 = 2, b = 25. Step 3: c = b / a = 25 / 400 = 1 / 16\n",
      "The answer is a, 1 / 16\n",
      "\n",
      "Question: the membership of a committee consists of 3 english teachers , 4 mathematics teachers , and 2 social studies teachers . if 2 committee members are to be selected at random to write the committee ’ s report , what is the probability that the two members selected will both be maths teachers ?\n",
      "Options: a ) 2 / 3 , b ) 1 / 3 , c ) 2 / 9 , d ) 1 / 24 , e ) 1 / 12\n",
      "Let's think step by step\n",
      "Step 1: probability of first member an english teacher = 3 / 9. Step 2: probability of second member an english teacher = 2 / 8.. Step 3: probability of both being english teacher = 3 / 9 x 2 / 8 = 1 / 12\n",
      "The answer is e, 1 / 12\n",
      "\n",
      "Question: the radius of a cylindrical vessel is 7 cm and height is 5 cm . find the whole surface of the cylinder ?\n",
      "Options: a ) 281 , b ) 528 , c ) 440 , d ) 767 , e ) 1981\n",
      "Let's think step by step\n",
      "Step 1: r = 7, h = 5; Step 2: 2 π r ( h + r ) = 2 * 22 / 7 * 7 ( 12 ) = 528\n",
      "The answer is b, 528\n",
      "\n",
      "Question: dan ’ s car gets 32 miles per gallon . if gas costs $ 4 / gallon , then how many miles can dan ’ s car go on $ 58 of gas ?\n",
      "Options: a ) 242 , b ) 353 , c ) 464 , d ) 575 , e ) 686\n",
      "Let's think step by step\n",
      "Step 1: 58 / 4 = 14.5 gallons. Step 2: 14.5 * 32 = 464 miles\n",
      "the answer is c, 464\n",
      "\n",
      "Question: average of 5 numbers is 20 , and the sum of 3 of the numbers is 48 , what is the average of the other 2 numbers ?\n",
      "Options: a ) 24 , b ) 34 , c ) 29 , d ) 35 , e ) 26\n",
      "Let's think step by step\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(prompt_full)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "1e3c97a3-a3d1-4776-9972-c7129e88c9d8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "num_q 200 correct 67 ratio 0.3350\n"
     ]
    }
   ],
   "source": [
    "_, _, _ = parse_pred_ans('outputs/dev_outputs_simple_explicit.txt')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0f117400-06ef-47d5-9e14-9494a492319f",
   "metadata": {},
   "source": [
    "## Semicolon"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "c97655fa-23dd-4f96-af97-ffef021b9d1d",
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt_simple_semicolon = open('MathQA/prompt_simple_semicolon.txt').read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "de2612c0-1ef7-4a4e-8f9e-a005e19286b7",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 200/200 [09:17<00:00,  2.79s/it]\n"
     ]
    }
   ],
   "source": [
    "i = 0\n",
    "with open('outputs/dev_outputs_simple_semicolon.txt', 'w') as fd:\n",
    "    for case in tqdm(dev_data_200):\n",
    "        q = case['Problem']\n",
    "        c = case['options']\n",
    "        a = case['Rationale']\n",
    "        cr = case['correct']\n",
    "    \n",
    "        ans_dict = get_ans_dict(c)\n",
    "            \n",
    "        a += ' ' + ans_dict[cr]\n",
    "        \n",
    "        prompt_q = '' \n",
    "        prompt_q += '\\nQuestion: ' + q + '\\n'\n",
    "        prompt_q += 'Options: ' + c + '\\n'\n",
    "        prompt_q += \"Let's think step by step\\n\"\n",
    "        \n",
    "        prompt_full = ''.join(prompt_simple_semicolon) + prompt_q\n",
    "        response = openai.Completion.create(model=\"text-davinci-002\", \n",
    "                                            prompt=prompt_full, \n",
    "                                            temperature=0, \n",
    "                                            max_tokens=256)\n",
    "        \n",
    "        ans_model = response['choices'][0]['text']\n",
    "        fd.write('%sA_model:\\n%s\\nA:\\n%s\\n\\n' % (prompt_q, ans_model, a))\n",
    "        i += 1\n",
    "        # if(i == 2): break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "872c4928-19a5-4e77-8618-9959b67b8a57",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "num_q 200 correct 74 ratio 0.3700\n"
     ]
    }
   ],
   "source": [
    "_, _, _ = parse_pred_ans('outputs/dev_outputs_simple_semicolon.txt')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4596dd4f-a615-4f61-8130-b062929e0223",
   "metadata": {},
   "source": [
    "## Exclamation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "aa4e53d8-7be5-4c47-8b34-51b780126af4",
   "metadata": {},
   "outputs": [],
   "source": [
    "prompt_simple_exclamation = open('MathQA/prompt_simple_exclamation.txt').read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "b52693b9-f65e-4409-b520-72d1d695125f",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 200/200 [09:44<00:00,  2.92s/it]\n"
     ]
    }
   ],
   "source": [
    "i = 0\n",
    "with open('outputs/dev_outputs_simple_exclamation.txt', 'w') as fd:\n",
    "    for case in tqdm(dev_data_200):\n",
    "        q = case['Problem']\n",
    "        c = case['options']\n",
    "        a = case['Rationale']\n",
    "        cr = case['correct']\n",
    "    \n",
    "        ans_dict = get_ans_dict(c)\n",
    "            \n",
    "        a += ' ' + ans_dict[cr]\n",
    "        \n",
    "        prompt_q = '' \n",
    "        prompt_q += '\\nQuestion: ' + q + '\\n'\n",
    "        prompt_q += 'Options: ' + c + '\\n'\n",
    "        prompt_q += \"Let's think step by step\\n\"\n",
    "        \n",
    "        prompt_full = ''.join(prompt_simple_exclamation) + prompt_q\n",
    "        response = openai.Completion.create(model=\"text-davinci-002\", \n",
    "                                            prompt=prompt_full, \n",
    "                                            temperature=0, \n",
    "                                            max_tokens=256)\n",
    "        \n",
    "        ans_model = response['choices'][0]['text']\n",
    "        fd.write('%sA_model:\\n%s\\nA:\\n%s\\n\\n' % (prompt_q, ans_model, a))\n",
    "        i += 1\n",
    "        # if(i == 2): break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "id": "2b88d421-f730-4c20-a537-d9b95394daa5",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Question: on a purchase of $ 130 , a store offered a payment plan consisting of a $ 30 down payment and 12 monthly payments of $ 10 each . what percent of the purchase price , to the nearest tenth of a percent , did the customer pay in interest by using this plan ?\n",
      "Options: a ) 23.5 % , b ) 24.5 % , c ) 23.1 % , d ) 25.5 % , e ) 26.5 %\n",
      "Let's think step by step\n",
      "12 * 10 + 30 = 150!!! ( 30 / 130 ) * 100 = 23.1\n",
      "The answer is c, 23.1 %\n",
      "\n",
      "Question: the ratio of three numbers is 1 : 3 : 4 and the sum of their squares is 2600 . the sum of the numbers is ?\n",
      "Options: a ) 10 , b ) 12 , c ) 15 , d ) 14 , e ) 9\n",
      "Let's think step by step\n",
      "let the numbers be x , 3 x , 4 x then , x^2 + 9x^2 + 16x^2 = 2600!!! 26 x^2 = 2600, x^2 = 100, x = 10\n",
      "The answer is a, 10\n",
      "\n",
      "Question: the average weight of 8 person ' s increases by 1.5 kg when a new person comes in place of one of them weighing 75 kg . what might be the weight of the new person ?\n",
      "Options: a ) 81 kg , b ) 85 kg , c ) 90 kg , d ) 100 kg , e ) 110 kg\n",
      "Let's think step by step\n",
      "total weight increased = ( 8 x 1.5 ) kg = 6 kg!!! weight of new person = ( 75 + 6 ) kg = 81 kg\n",
      "The answer is a, 81 kg\n",
      "\n",
      "Question: how many positive integers less than 60 have a reminder 01 when divided by 3 ?\n",
      "Options: a ) 13 , b ) 14 , c ) 15 , d ) 16 , e ) 20\n",
      "Let's think step by step\n",
      "1 also gives the remainder of 1 when divided by 3!!! so , there are total of 20 numbers\n",
      "The answer is e, 20\n",
      "\n",
      "Question: 8 is 2 % of a , and 2 is 8 % of b . c equals b / a . what is the value of c ?\n",
      "Options: a ) 1 / 16 , b ) 1 / 4 , c ) 1 , d ) 4 , e ) 8\n",
      "Let's think step by step\n",
      "2a / 100 = 8, a = 400!!! 8b / 100 = 2, b = 25!!! c = b / a = 25 / 400 = 1 / 16\n",
      "The answer is a, 1 / 16\n",
      "\n",
      "Question: the membership of a committee consists of 3 english teachers , 4 mathematics teachers , and 2 social studies teachers . if 2 committee members are to be selected at random to write the committee ’ s report , what is the probability that the two members selected will both be maths teachers ?\n",
      "Options: a ) 2 / 3 , b ) 1 / 3 , c ) 2 / 9 , d ) 1 / 24 , e ) 1 / 12\n",
      "Let's think step by step\n",
      "probability of first member an english teacher = 3 / 9!!! probability of second member an english teacher = 2 / 8!!! probability of both being english teacher = 3 / 9 x 2 / 8 = 1 / 12\n",
      "The answer is e, 1 / 12\n",
      "\n",
      "Question: the radius of a cylindrical vessel is 7 cm and height is 5 cm . find the whole surface of the cylinder ?\n",
      "Options: a ) 281 , b ) 528 , c ) 440 , d ) 767 , e ) 1981\n",
      "Let's think step by step\n",
      "r = 7, h = 5!!! 2 π r ( h + r ) = 2 * 22 / 7 * 7 ( 12 ) = 528\n",
      "The answer is b, 528\n",
      "\n",
      "Question: dan ’ s car gets 32 miles per gallon . if gas costs $ 4 / gallon , then how many miles can dan ’ s car go on $ 58 of gas ?\n",
      "Options: a ) 242 , b ) 353 , c ) 464 , d ) 575 , e ) 686\n",
      "Let's think step by step\n",
      "58 / 4 = 14.5 gallons!!! 14.5 * 32 = 464 miles\n",
      "the answer is c, 464\n",
      "\n",
      "Question: amar takes as much time in running 22 meters as a car takes in covering 58 meters . what will be the distance covered by amar during the time the car covers 2.6 km ?\n",
      "Options: a ) 700 m , b ) 978 m , c ) 970 m , d ) 986 m , e ) 640 m\n",
      "Let's think step by step\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(prompt_full)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "ece74701-67d2-4b19-96f2-da8aff145bd6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "num_q 200 correct 85 ratio 0.4250\n"
     ]
    }
   ],
   "source": [
    "_, _, _ = parse_pred_ans('outputs/dev_outputs_simple_exclamation.txt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "532634a9-7f33-490e-a857-4e9fedd22d8b",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
